{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp symbolic_prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Symbolic Prediction\n",
    "\n",
    "> Class to generate good quality evading instances that are different from original instance by exactly 1 feature."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import IPython, graphviz\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from copy import deepcopy\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from tree_evasion.core import *\n",
    "from tree_evasion.tree import *\n",
    "\n",
    "SEED = 41\n",
    "np.random.seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "_all_ = ['SymbolicInstance', 'SymbolicPredictionSolver', 'CoordinateDescent']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class SymbolicInstance:\n",
    "    def __init__(self, constraints):\n",
    "        self.constraints = constraints  \n",
    "        self.x_prime_constraints = {}\n",
    "        self.k = None\n",
    "    \n",
    "    def diff_instances(self):\n",
    "        ck  = list(set(self.x_prime_constraints.keys()) & set(self.constraints.keys()))\n",
    "        res = 0\n",
    "        \n",
    "        for k in ck:\n",
    "            if (k in self.x_prime_constraints) and (k in self.constraints):\n",
    "                if self.constraints[k] != self.x_prime_constraints[k]:\n",
    "                    res += 1\n",
    "        return res\n",
    "    \n",
    "    def is_feasible(self, predicate, res):\n",
    "        feat, threshold = predicate\n",
    "        \n",
    "        d = self.diff_instances()\n",
    "        \n",
    "        if d == 1:\n",
    "            if feat not in self.constraints: return False\n",
    "            if (threshold > self.constraints[feat][0]) and (threshold <= self.constraints[feat][1]): return True\n",
    "            return False\n",
    "        else:\n",
    "            if feat not in self.constraints: return False\n",
    "        \n",
    "        return True\n",
    "    \n",
    "    def update(self, predicate, res):\n",
    "        feat, threshold = predicate\n",
    "        self.k = feat\n",
    "        \n",
    "        if res:\n",
    "            self.x_prime_constraints[feat] = [-np.inf, threshold]\n",
    "        else:\n",
    "            self.x_prime_constraints[feat] = [threshold, np.inf]\n",
    "    \n",
    "    def is_changed(self): \n",
    "        return self.diff_instances() == 1\n",
    "    \n",
    "    def get_perturbation(self): return (self.k, self.x_prime_constraints[self.k])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class SymbolicPredictionSolver:\n",
    "    def solve(self, tree, index, s):\n",
    "        self.l = []\n",
    "        self.symbolic_prediction(tree, index, s)\n",
    "        return self.l\n",
    "\n",
    "    def symbolic_prediction(self, tree, index, s):\n",
    "        if tree.is_leaf(index):\n",
    "            if s.is_changed():        \n",
    "                k, half_open_interval = s.get_perturbation()\n",
    "                self.l.extend([[k, half_open_interval, tree.prediction(index)]])\n",
    "        else:\n",
    "            if s.is_feasible(tree.predicate(index), True):\n",
    "                s_t = deepcopy(s)\n",
    "                s_t.update(tree.predicate(index), True)\n",
    "                self.symbolic_prediction(tree, tree.get_left_child_index(index), s_t)\n",
    "            if s.is_feasible(tree.predicate(index), False):\n",
    "                s.update(tree.predicate(index), False)\n",
    "                self.symbolic_prediction(tree, tree.get_right_child_index(index), s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class CoordinateDescent:\n",
    "    @classmethod\n",
    "    def get_pairs(cls, clf, Xte):\n",
    "        pairs = []\n",
    "\n",
    "        for i in tqdm(range(2)):\n",
    "            forest_prediction_intervals = []\n",
    "            x = Xte[i:i+1]\n",
    "\n",
    "            # if this instance is correctly predicted by the classifier or not\n",
    "            p = clf.predict(x)[0]\n",
    "\n",
    "            if p != yte[i]:\n",
    "                continue\n",
    "\n",
    "            for tidx in range(len(clf.estimators_)):\n",
    "                t = Tree(clf.estimators_[tidx].tree_) \n",
    "\n",
    "                x_constraints = t.constraints(x)\n",
    "                s = SymbolicInstance(x_constraints)\n",
    "                solver = SymbolicPredictionSolver()\n",
    "                perturbations = solver.solve(t, 0, s)\n",
    "\n",
    "                # index of the leaf node for x\n",
    "                x_path = clf.estimators_[tidx].tree_.decision_path(x.astype(np.float32)).toarray().ravel()\n",
    "                leaf_node_index = np.where(x_path == 1)[0][-1]\n",
    "                pred = t.prediction(leaf_node_index)\n",
    "\n",
    "                prediction_intervals = []\n",
    "                for index, p in enumerate(perturbations):\n",
    "                    prediction_intervals.append([p[0], p[1], p[2] - pred])\n",
    "\n",
    "                forest_prediction_intervals.extend(prediction_intervals)\n",
    "\n",
    "            max_interval = sorted(forest_prediction_intervals, key=lambda x: x[2], reverse=True)[0]\n",
    "            class_ = 0 if clf.predict(x)[0] == '2' else 1\n",
    "\n",
    "            x_prime = Xte[i].copy()\n",
    "            x_prime[max_interval[0]] = np.random.uniform(0 if max_interval[1][0] == -np.inf else max_interval[1][0], \n",
    "                                                         1 if max_interval[1][1] == np.inf else max_interval[1][1])\n",
    "\n",
    "            perturbed_class = 0 if clf.predict(x_prime.reshape(1, -1))[0] == '2' else 1\n",
    "\n",
    "            if class_ != perturbed_class:\n",
    "                print(f'class: {class_}, perturbed_class: {perturbed_class}')\n",
    "                pairs.append((x, x_prime))\n",
    "\n",
    "        return pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xtr, Xva, Xte, ytr, yva, yte = get_mnist_dataset(SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 120 ms, sys: 25.1 ms, total: 145 ms\n",
      "Wall time: 144 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(max_depth=4, n_estimators=5, n_jobs=-1, random_state=41)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "clf = RandomForestClassifier(n_estimators=5, max_depth=4, random_state=SEED, n_jobs=-1)\n",
    "clf.fit(Xtr, ytr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9585398828301036\n"
     ]
    }
   ],
   "source": [
    "# performance on the holdout set\n",
    "print(clf.score(Xva, yva))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:00<00:00,  3.16it/s]\n"
     ]
    }
   ],
   "source": [
    "pairs = CoordinateDescent.get_pairs(clf, Xte)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeAAAAD6CAYAAAB57pTcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAAaa0lEQVR4nO3de9BcdZng8e8jNwuMQrhGEkBJYGVxBlkGmR3UES/LbQxUiYCjohtAq6RWShx1FZkwZGYYV0DdHZnlJoiKSLxRuLOCjCBsOQhExCBMQOQWXhIuAgnFPc/+cc67+xLec7rffrv710m+n6qu9Hue0+c8fdLPefrc+kRmIkmShusVpROQJGlDZAOWJKkAG7AkSQXYgCVJKsAGLElSATZgSZIKsAGro4i4JiKOLZ2HVEpE/FNEfGEar/9wRFzfz5zWmv4/R8QxE/5eFBGPRMRDEbFTRKyOiI16mO4uEZERsXF/MxbYgPsuIt4fETfVH/ixujD2H4G87omId5bOY6oiYr+IuCoiHouIhyPisoiYVTovrR/qunguIrZZa/iv6sazC0BmfiwzTyuSZBcy86DMvAggInYCTgL2yMwdMvO+zHxVZr5YNstmEbFR/aXhwYhYVS//LUvnNWg24D6KiE8CXwb+Dtge2An4GjC/h2m97BvnBvotdCvgHGAXYGdgFfD1kglpvfN74OjxPyLijcDm5dKZtp2ARzNzZelEpuBU4D8Cfwq8Gvgg8EzRjIYhM3304QG8BlgNHNEyzmZUDfrB+vFlYLM69ufAA8BngIeAi4GFwGLgm8CTwLH1fM4HxoDlwCJgownzOA64napR/RbYu57WGuDpOsdPN+Q3H7ilntfvgAPr4dcAx9bPdwX+BXgUeAT4FrDlhGl8ps5rFfBvwDvq4fsCN9XTXgGc2eNy3htYVfr/28f68QDuAU4Gbpww7EvA54EEdqmHXQgsqp9vA1wBPA48BlwHvKKOzQG+Dzxc18j/qId/GLh+wjy+Atxf18PNwFsmxCatFeCV9brg0XreNwLb17Fr6vXDO+s6X1PX+oVUX14T2Lget3EdAmxUv/9HgLuBj0987STL77P1umJ8fXP4hNhc4FrgiXp6lzZMY6s6111Lfx6G/XALuH/+lKpAftAyzueB/YC9gD+mKrSTJ8R3AGZSbekdXw+bT9WEt6RqdhcCL1B9uN8EvJuq8IiII6ia9oeovkW+h+qb8AeB+4C/yGpX1BfXTiwi9gW+AfxVPa+3Uq2cXjYq8PfAa4E3UK1wFtbT2B04AfiTzJwB/KcJ0/gK8JXMfDVVE//uhHnfGhHvn3SJvdxbgdu6HFfqxr8Cr46IN9THSY+ianRNTqL6srwt1Z6uzwFZv/YK4F6qprcj8J2GadxItR6YCXwbuCwiXlnHmmrlGKrmOQfYGvgYVbP9fzLzp8BBwIN1rX94knlfSMM6hOoL/KH18H2A97YsB6ia71vqvE4FvjnhENFpwJVUDXY28N8bpvHGOp/31sesl0XExzvMd71gA+6frYFHMvOFlnH+EvibzFyZmQ9TfWA/OCG+BvjrzHw2M8cL6xeZ+cPMXEPVVA8GTszMp7LaxXQW1QoDqiL6YmbemJW7MvPeLvNfAFyQmVdl5prMXJ6Zd6w9Uj3Nq+ocHwbOBN5Wh1+k2srfIyI2ycx7MvN3dex5YG5EbJOZqzPzXydM848y89udEoyIPwJOofqSIPXTxVRfXN9FtQdpecu4zwOzgJ0z8/nMvC6rTbl9qb6Y/lVdn89k5qQnXmXmNzPz0cx8ITPPoKqb3SdMf7JaeZ5qPTM3M1/MzJsz88mpvMmI2J72dcj7gC9n5v2Z+RjVl+1GmXlZZj5YrzMuBe6sl8N4vjsDr21bFlTN+TXAbsDrqJr+woh411Te27rIBtw/jwLbdDhO+1qqb8fj7q2HjXs4M9c+7nH/hOc7A5sAYxHxeEQ8DvxPYLs6PofqG2kvunptRGwfEd+JiOUR8STVlsI2UDVn4ESqLeKV9Xjj728BVYHdERE3RsShU0kuIuYC/wx8IjOvm8prpS5cDLyfalfxNzqM+9+Au4ArI+LuiPhsPXwOcG+HL+EARMSnIuL2iHiiruPXUNcRzbVyMfAT4Dv1yUpfjIhNun+LQOd1yGt56Tqn9Qt8RHwoIm6ZMK09J7yPT1PtMftlRNwWEf+5YTLjGxt/k5lPZ+atVHsODp7aW1v32ID75xfAs8BhLeM8SFUA43aqh42b7NZUE4fdX89jm8zcsn68OjP//YT4rg3z7nTbq7bXTvR39bTeWO8i+wBVkVUzyfx2Zu5P9T4T+Id6+J2ZeTRVof8DsDgituhifkTEzsBPgdMy8+JuXiNNRb2n6PdUK/3vdxh3VWaelJmvpzrM88mIeAdVDe3U6WTJiHgLVXN6H7BVZm5JdZw06ulPWiv11vapmbkH1QlLh1JttU9Fp3XIGNUXiXE7tbyPnYFzqQ47bV2/j6UT3sdDmXlcZr4W+CjwtfqL9Npurf+duI7aIG7TZwPuk8x8gmr36D9GxGERsXlEbBIRB0XE+DHXS4CTI2Lb+rKHU2g/1rT2PMaojqmcERGvjohXRMSuETG+C/g84FMR8R+iMrcuEqhO5nh9y+TPBz4SEe+op7tjRPy7ScabQXXCxBMRsSMTdgdHxO4RcUBEbEZ1BuP4ySBExAciYtt6V/rj9UvWdHrP9Tz+hepkln/qNL40DQuAAzLzqbaRIuLQuraCqnG+SPVZ/iVVAzs9IraIiFdGxJ9NMokZVMc8HwY2johTqA4vjU9/0lqJiLdHxBvrY81PUu3i7VhDE3WxDvku8F8iYnZEbEV1klWTLaga5cN13h+h2gIefx9HRMTs+s8/1OO+LN/6MNV1wOcjYrOIeAPVLvErpvLe1kU24D6qj+V8kurEqoepvm2eAPywHmUR1dmNtwK/AZbUw6biQ8CmVGcc/oHqBK1Z9fwvA/6W6qSOVfV8Z9av+3uq5v94RHxqktx/CXyE6njQE1RnL+689nhUx633rsf5MS/dWtgMOJ3qjMeHqL7B/9c6diBwW0SspjrJ5Kjx49z17qm/bHi/x1J9cVgY1bXVq+tpSH2Vmb/LzJu6GHUe1R6Z1VR7vr6WmT/L6jrbv6A6uek+qhO1jpzk9T8B/jewjGoX7zO8dLdvU63sQFXvT1Idp76Warf0VDWuQ6i2aH8C/Jpq/dS4NyAzfwucQbUMVlCdTPV/JozyJ8AN9fu4nOrw0d0Nkzuaan3zKNV65QuZeXUP722dEtW5A5IkaZjcApYkqQAbsCRJBdiAJUkqwAYsSVIBNmBJkgqY1t11IuJAqtPkNwLOy8zTO4zvKddSdx7JzG2HOcOp1LO1LHWtsZZ73gKuLwb/R6of/t4DODoi9uh1epJeotvf8O4L61kamMZans4u6H2BuzLz7sx8juq3O6d831tJI8F6loZsOg14R1766y0P1MMkrXusZ2nIpnUMuBsRcTz//962ktZR1rLUX9NpwMt56V0zZjPJPTQz8xzgHPDEDWmEdaxna1nqr+nsgr4RmBcRr4uITanuXnF5f9KSNGTWszRkPW8BZ+YLEXEC1Z0zNgIuyMzb+paZpKGxnqXhG+rdkNxtJXXt5szcp3QSTaxlqWuNtewvYUmSVIANWJKkAmzAkiQVYAOWJKkAG7AkSQUM/Jew1D8//OEPG2Pz5zf/bO+aNWsaY0uWLOk5n0WLFjXGrr322sbY008/3TrdZ599tuecpHWBtSxwC1iSpCJswJIkFWADliSpABuwJEkF2IAlSSrABixJUgE2YEmSCvBuSCNk6623bo0vXbq0Mbbddts1xgb1fxwRjbFly5Y1xr761a+2Tvfss8/uOaf1iHdDWodZyxVrGfBuSJIkjRYbsCRJBdiAJUkqwAYsSVIBNmBJkgqwAUuSVIC3IxwhBxxwQGt82223bYxddtll/U4HgHnz5jXG9tprr55e97GPfax1ns8880xj7Otf/3rra6VRYC1XrOV2bgFLklSADViSpAJswJIkFWADliSpABuwJEkF2IAlSSpgWndDioh7gFXAi8ALne7e4h1U2m2++eat8ZkzZzbGHnjggX6nA8CMGTMaY+95z3saYwcffHBj7Mgjj+w5n0suuaQxdtpppzXG2u7oMqKGfjekqdSztdzOWu7MWu7PdcBvz8xH+jAdSeVZz9KQuAtakqQCptuAE7gyIm6OiOP7kZCkYqxnaYimuwt6/8xcHhHbAVdFxB2Z+fOJI9SFbDFLo6+1nq1lqb+mtQWcmcvrf1cCPwD2nWScczJzn2GfUCJpajrVs7Us9VfPDTgitoiIGePPgXcDS/uVmKThsZ6l4ev5MqSIeD3Vt2SodmV/OzP/tsNrvHRBHd13332NsdmzZzfGli9f3hg76KCDWue5dOnI9ZqhXoY01Xq2ltUNaxkYxGVImXk38Mc9pyRpZFjP0vB5GZIkSQXYgCVJKsAGLElSATZgSZIKsAFLklSADViSpAL6cTckqa/OPffcxtjChQsbY7NmzWqMtd1SDUby2kFpnWctt3MLWJKkAmzAkiQVYAOWJKkAG7AkSQXYgCVJKsAGLElSAT3fjrCnmXkLM03THXfc0RibO3duY2zx4sWt0z3qqKN6zmlAhno7wqmyljVd1rJbwJIkFWEDliSpABuwJEkF2IAlSSrABixJUgE2YEmSCvAyJK1TZs+e3Ri76qqrenodwG677dYYGxsb65xY/3kZktZr1rJbwJIkFWEDliSpABuwJEkF2IAlSSrABixJUgE2YEmSCti40wgRcQFwKLAyM/esh80ELgV2Ae4B3peZfxhcmlLloYceaozdddddjbFOly5sKKxnjQprubst4AuBA9ca9lng6sycB1xd/y1p9F2I9SyNhI4NODN/Djy21uD5wEX184uAw/qblqRBsJ6l0dHrMeDtM3P8J0UeArbvUz6Shs96lgroeAy4k8zMtp+li4jjgeOnOx9Jg9dWz9ay1F+9bgGviIhZAPW/K5tGzMxzMnOfUf5dW2kD11U9W8tSf/XagC8HjqmfHwP8qD/pSCrAepYK6NiAI+IS4BfA7hHxQEQsAE4H3hURdwLvrP+WNOKsZ2l0dDwGnJlHN4Te0edcpI4OOeSQxthBBx3UGFu8eHHrdAvdpmzorGeNCmvZX8KSJKkIG7AkSQXYgCVJKsAGLElSATZgSZIKsAFLklTAtH+KUhqmJUuW9PS6ZcuW9TkTrS9uuOGGxtib3/zmIWayYbGW3QKWJKkIG7AkSQXYgCVJKsAGLElSATZgSZIKsAFLklSAlyFpnbJgwYLGWNtlDYsWLRpEOloPeKlRGdayW8CSJBVhA5YkqQAbsCRJBdiAJUkqwAYsSVIBNmBJkgpY7y9D2njj5rd47LHHNsa+8IUvtE53hx12aIxFRGMsM1un2+bwww9vjL3tbW/raZqXXHJJa3zFihWNsfvvv7+neW6++eat8U9/+tONsVNOOaUxdvrppzfGnnvuuc6JaaRZy+2s5XWPW8CSJBVgA5YkqQAbsCRJBdiAJUkqwAYsSVIBNmBJkgqwAUuSVEB0upYtIi4ADgVWZuae9bCFwHHAw/Von8vM/9VxZhG9XzjXozlz5jTGfv/73w9knm3XDl5xxRWNsb333rt1ur/61a8aY4ccckhjbDrXK7Y5+eSTG2MPPvhgY6zt/wTg1FNPbYy1LdsjjzyyMbZ48eLWeY6gmzNzn35PtF/1bC1by2Atd6mxlrvZAr4QOHCS4Wdl5l71o2PzlTQSLsR6lkZCxwacmT8HHhtCLpIGzHqWRsd0jgGfEBG3RsQFEbFV00gRcXxE3BQRN01jXpIGq2M9W8tSf/XagM8GdgX2AsaAM5pGzMxzMnOfQRzPktQXXdWztSz1V08NODNXZOaLmbkGOBfYt79pSRoW61kqo6cGHBGzJvx5OLC0P+lIGjbrWSqjm8uQLgH+HNgGWAH8df33XkAC9wAfzcyxjjMrcOnCZptt1hhru4zg7W9/e8/zfPrppxtjP/7xjxtjS5YsaZ1u26n5W2yxRWNs7ty5rdPt1aBu1dbrPJctW9YYO+KII1qnu3TpyPWcQV2G1Jd6tpat5UHOc0Op5Y73A87MoycZfP60U5I0dNazNDr8JSxJkgqwAUuSVIANWJKkAmzAkiQVYAOWJKmAjmdBr+ueffbZxthjj/X+k7j77bdfY2z58uWNsbGxjldrNTr77LMbYwcccEBjrO3Shd122611ngsWLOic2Ig466yzGmNtlzVo3WAtW8uwftWyW8CSJBVgA5YkqQAbsCRJBdiAJUkqwAYsSVIBNmBJkgpY7y9Dmj9/fmPsve99b2PszjvvbJ3u7rvv3hhbsWJFY2zOnDmt0+3VjBkzGmPHHXdcY2zevHk9z7PtbibPPfdcY2z16tWt0910000bY5dffnlj7JprrukpH60brGVruVM+6xq3gCVJKsAGLElSATZgSZIKsAFLklSADViSpAJswJIkFWADliSpgMjM4c0sYngzq2233XaNsSuvvLIxtueee/Y8z7Zr6ga1vEvM8/nnn2+MXXTRRY2xRYsWtU536623boz9+te/7pzY+uHmzNyndBJNrGVrGazlLjXWslvAkiQVYAOWJKkAG7AkSQXYgCVJKsAGLElSATZgSZIK6HgZUkTMAb4BbA8kcE5mfiUiZgKXArsA9wDvy8w/dJjW0C9daLPbbrs1xq6//vrW186cObMxtj5durBs2bLG2Ac+8IHG2JIlS3qep4ABXYbUr3q2lq1ldW1alyG9AJyUmXsA+wEfj4g9gM8CV2fmPODq+m9Jo816lkZExwacmWOZuaR+vgq4HdgRmA+MX6F9EXDYgHKU1CfWszQ6Np7KyBGxC/Am4AZg+8wcq0MPUe3Smuw1xwPHTyNHSQMw1Xq2lqX+6vokrIh4FfA94MTMfHJiLKsDE5MenMjMczJzn1H+WT1pQ9NLPVvLUn911YAjYhOqYv1WZn6/HrwiImbV8VnAysGkKKmfrGdpNHRswFGdknc+cHtmnjkhdDlwTP38GOBH/U9PUj9Zz9Lo6OYypP2B64DfAGvqwZ+jOm70XWAn4F6qyxYe6zCtkbp0oU3bnVcAzjvvvMZY2+n+hxxySGNs3rx5rfM899xzG2NPPfVUY2xsbKwxdumll7bOc9WqVY2xJ554ovW1mpZBXYbUl3q2lq1lda2xljuehJWZ1wNNF6a9YzpZSRou61kaHf4SliRJBdiAJUkqwAYsSVIBNmBJkgqwAUuSVEDHy5D6OrN16NIFqbCBXIbUL9ay1LVp3Q1JkiT1mQ1YkqQCbMCSJBVgA5YkqQAbsCRJBdiAJUkqwAYsSVIBNmBJkgqwAUuSVIANWJKkAmzAkiQVYAOWJKkAG7AkSQXYgCVJKsAGLElSATZgSZIKsAFLklSADViSpAJswJIkFWADliSpABuwJEkFdGzAETEnIn4WEb+NiNsi4hP18IURsTwibqkfBw8+XUm9spal0bJxF+O8AJyUmUsiYgZwc0RcVcfOyswvDS49SX1kLUsjpGMDzswxYKx+vioibgd2HHRikvrLWpZGy5SOAUfELsCbgBvqQSdExK0RcUFEbNXv5CQNhrUsldd1A46IVwHfA07MzCeBs4Fdgb2ovlWf0fC64yPipoi4afrpSpoua1kaDZGZnUeK2AS4AvhJZp45SXwX4IrM3LPDdDrPTBLAzZm5T78nai1LQ9dYy92cBR3A+cDtEws2ImZNGO1wYOl0s5Q0ONayNFq6OQv6z4APAr+JiFvqYZ8Djo6IvYAE7gE+OoD8JPWPtSyNkK52QfdtZu62kro1kF3Q/WItS13rfRe0JEnqPxuwJEkF2IAlSSrABixJUgE2YEmSCrABS5JUgA1YkqQCbMCSJBVgA5YkqQAbsCRJBdiAJUkqwAYsSVIBNmBJkgro5naE/fQIcO+Ev7eph40K82k3avnA6OXUr3x27sM0BslanrpRy8l82g28lod6O8KXzTziplG65Zr5tBu1fGD0chq1fIZl1N73qOUDo5eT+bQbRj7ugpYkqQAbsCRJBZRuwOcUnv/azKfdqOUDo5fTqOUzLKP2vkctHxi9nMyn3cDzKXoMWJKkDVXpLWBJkjZIRRpwRBwYEf8WEXdFxGdL5LBWPvdExG8i4paIuKlQDhdExMqIWDph2MyIuCoi7qz/3apwPgsjYnm9nG6JiIOHmM+ciPhZRPw2Im6LiE/Uw4sso5Z8ii2jUqznl81/pGq5Jacin9VRq+UOOQ10GQ19F3REbAQsA94FPADcCBydmb8daiIvzekeYJ/MLHYNWkS8FVgNfCMz96yHfRF4LDNPr1dsW2XmZwrmsxBYnZlfGkYOa+UzC5iVmUsiYgZwM3AY8GEKLKOWfN5HoWVUgvU86fxHqpZbclpIgc/qqNVyh5wGWs8ltoD3Be7KzLsz8zngO8D8AnmMlMz8OfDYWoPnAxfVzy+i+kCUzKeYzBzLzCX181XA7cCOFFpGLflsaKzntYxaLbfkVMSo1XKHnAaqRAPeEbh/wt8PUH7FlcCVEXFzRBxfOJeJts/Msfr5Q8D2JZOpnRARt9a7tIa6G21cROwCvAm4gRFYRmvlAyOwjIbIeu5O8c9pg6Kf1VGr5UlyggEuI0/CquyfmXsDBwEfr3fXjJSsjhWUPmX9bGBXYC9gDDhj2AlExKuA7wEnZuaTE2MlltEk+RRfRhrteh6RWobCn9VRq+WGnAa6jEo04OXAnAl/z66HFZOZy+t/VwI/oNqtNgpW1Mcmxo9RrCyZTGauyMwXM3MNcC5DXk4RsQlVcXwrM79fDy62jCbLp/QyKsB67s5I1TKU/ayOWi035TToZVSiAd8IzIuI10XEpsBRwOUF8gAgIraoD7oTEVsA7waWtr9qaC4HjqmfHwP8qGAu40Ux7nCGuJwiIoDzgdsz88wJoSLLqCmfksuoEOu5OyNVy1DuszpqtdyW08CXUWYO/QEcTHXm5O+Az5fIYUIurwd+XT9uK5UPcAnVLo7nqY6jLQC2Bq4G7gR+CswsnM/FwG+AW6mKZdYQ89mfapfUrcAt9ePgUsuoJZ9iy6jUw3p+WQ4jVcstORX5rI5aLXfIaaDLyF/CkiSpAE/CkiSpABuwJEkF2IAlSSrABixJUgE2YEmSCrABS5JUgA1YkqQCbMCSJBXwfwG1byCpOn9lcgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# fig, ax = plt.subplots(1, 2, figsize=(8, 8))\n",
    "\n",
    "# ax[0].imshow(pairs[0][0].reshape(28, 28), cmap='gray')\n",
    "# ax[0].set_title('Correct class: 2')\n",
    "# ax[1].imshow(pairs[0][1].reshape(28, 28), cmap='gray')\n",
    "# ax[1].set_title('Misclassified as 6');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeAAAAD6CAYAAAB57pTcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAAY40lEQVR4nO3df/RcdX3n8efbgHiKQH4gMUAgmFCFxRJdQFjwB9WwEWnBc5AVW0UWiZwDZ+EIRRatxQIttYLgbhWxsETwd4GK7LaCgJXuEYWwKb9C+RmEkJCEGJL01ErCe/+497s7hO+dme/3OzOfSfJ8nPM9me/93Ln3PTfzntfMvXe+NzITSZI0WK8pXYAkSdsiA1iSpAIMYEmSCjCAJUkqwACWJKkAA1iSpAIMYHUUET+JiE+UrkMqJSKujIg/nsD9Px4R/9jLmjZb/t9FxEktv18UEasjYkVE7BURGyJi0jiWOysiMiK2623FAgO45yLiIxFxb/2EX143xhFDUNfSiHhf6TrGKiIOjYjbImJNRKyKiO9HxIzSdWnrUPfFbyJi182m/586eGYBZOZpmXlhkSK7kJnvz8yFABGxF3A2sH9mvjEzf5mZr8/MTWWrbBYRk+o3Dc9FxPp6+08uXVe/GcA9FBGfAi4H/gyYDuwFfAU4dhzLetU7zm30XegU4CpgFrA3sB74HyUL0lbnKeDEkV8i4q3Ab5UrZ8L2Al7IzJWlCxmDzwP/ATgM2Bn4KPDrohUNQmb604MfYBdgA/ChNvPsQBXQz9U/lwM71GPvAZ4FPg2sAK4DLgD+BrgeWAd8ol7P1cByYBlwETCpZR2nAkuoguph4O31sl4G/rWu8dyG+o4FFtfregKYX0//CfCJ+vZs4A7gBWA18E1gcssyPl3XtR74Z+C99fRDgHvrZT8PXDbO7fx2YH3p/29/to4fYCnwWeCelmlfBD4DJDCrnnYtcFF9e1fgFmAtsAa4C3hNPTYTuBFYVffIf6+nfxz4x5Z1XAE8U/fDIuCdLWOj9grwuvq14IV63fcA0+uxn9SvD++r+/zlutevpXrzmsB29byNryHApPrxrwaeBE5vve8o2++8+rVi5PXmgy1jc4B/AF6sl/fdhmVMqWudXfr5MOgfPwH3zmFUDXJTm3k+AxwKzAUOpGq0z7aMvxGYSvVJb0E97ViqEJ5MFXbXAhupntxvA46iajwi4kNUof0xqneRv0/1TvijwC+B38tqV9QXNi8sIg4BvgH8Ub2ud1G9OL1qVuDPgd2B/ahecC6ol/Fm4Azg4MzcCfiPLcu4ArgiM3emCvHvtaz7/oj4yKhb7NXeBTzU5bxSN+4Gdo6I/erjpB+mCromZ1O9WX4D1Z6u84Gs73sL8DRV6O0BfKdhGfdQvQ5MBb4FfD8iXlePNfXKSVThOROYBpxGFbb/T2b+GHg/8Fzd6x8fZd3X0vAaQvUG/ph6+kHA8W22A1Th+866rs8D17ccIroQuJUqYPcE/lvDMt5a13N8fcz60Yg4vcN6twoGcO9MA1Zn5sY28/wB8KeZuTIzV1E9YT/aMv4y8CeZ+W+ZOdJYP8vMv83Ml6lC9WjgrMz8l6x2MX2J6gUDqib6Qmbek5XHM/PpLus/BbgmM2/LzJczc1lmPrL5TPUyb6trXAVcBry7Ht5E9Sl//4jYPjOXZuYT9dhLwJyI2DUzN2Tm3S3L/J3M/FanAiPid4DPUb1JkHrpOqo3rvOo9iAtazPvS8AMYO/MfCkz78rqo9whVG9M/6juz19n5qgnXmXm9Zn5QmZuzMxLqfrmzS3LH61XXqJ6nZmTmZsyc1FmrhvLg4yI6bR/DTkBuDwzn8nMNVRvthtl5vcz87n6NeO7wGP1dhipd29g93bbgiqcdwF+G9iHKvQviIh5Y3lsWyIDuHdeAHbtcJx2d6p3xyOerqeNWJWZmx/3eKbl9t7A9sDyiFgbEWuBrwG71eMzqd6RjkdX942I6RHxnYhYFhHrqD4p7ApVOANnUX0iXlnPN/L4TqFqsEci4p6IOGYsxUXEHODvgDMz866x3FfqwnXAR6h2FX+jw7x/CTwO3BoRT0bEefX0mcDTHd6EAxAR50TEkoh4se7jXaj7iOZeuQ74EfCd+mSlL0TE9t0/RKDza8juvPI1p+0b+Ij4WEQsblnWAS2P41yqPWa/iIiHIuI/Nyxm5MPGn2bmv2bm/VR7Do4e20Pb8hjAvfMz4N+A49rM8xxVA4zYq542YrRLU7VOe6Zex66ZObn+2Tkz/13L+OyGdXe67FW7+7b6s3pZb613kf0hVZNVK8n8VmYeQfU4E/iLevpjmXkiVaP/BfA3EbFjF+sjIvYGfgxcmJnXdXMfaSzqPUVPUb3o39hh3vWZeXZmvonqMM+nIuK9VD20V6eTJSPinVThdAIwJTMnUx0njXr5o/ZK/Wn785m5P9UJS8dQfWofi06vIcup3kiM2KvN49gb+DrVYadp9eN4sOVxrMjMUzNzd+CTwFfqN9Kbu7/+t/U1apu4TJ8B3COZ+SLV7tG/iojjIuK3ImL7iHh/RIwcc/028NmIeEP9tYfP0f5Y0+brWE51TOXSiNg5Il4TEbMjYmQX8F8D50TEv4/KnLpJoDqZ401tFn81cHJEvLde7h4R8ZZR5tuJ6oSJFyNiD1p2B0fEmyPidyNiB6ozGEdOBiEi/jAi3lDvSl9b3+XlTo+5XscdVCezXNlpfmkCTgF+NzP/pd1MEXFM3VtBFZybqJ7Lv6AKsEsiYseIeF1EHD7KInaiOua5CtguIj5HdXhpZPmj9kpEHBkRb62PNa+j2sXbsYdadfEa8j3gv0TEnhExheokqyY7UgXlqrruk6k+AY88jg9FxJ71r7+q531VvfVhqruAz0TEDhGxH9Uu8VvG8ti2RAZwD9XHcj5FdWLVKqp3m2cAf1vPchHV2Y33Aw8A99XTxuJjwGupzjj8FdUJWjPq9X8fuJjqpI719Xqn1vf7c6rwXxsR54xS+y+Ak6mOB71Idfbi3pvPR3Xc+u31PP+TV35a2AG4hOqMxxVU7+D/az02H3goIjZQnWTy4ZHj3PXuqT9oeLyfoHrjcEFU363eUC9D6qnMfCIz7+1i1n2p9shsoNrz9ZXMvDOr79n+HtXJTb+kOlHrP41y/x8Bfw88SrWL99e8crdvU6+8karf11Edp/4Hqt3SY9X4GkL1ifZHwD9RvT417g3IzIeBS6m2wfNUJ1P975ZZDgZ+Xj+Om6kOHz3ZsLgTqV5vXqB6XfnjzLx9HI9tixLVuQOSJGmQ/AQsSVIBBrAkSQUYwJIkFWAAS5JUgAEsSVIBE7q6TkTMpzpNfhLw15l5SYf5PeVa6s7qzHzDIFc4ln62l6WuNfbyuD8B118G/yuqP/y9P3BiROw/3uVJeoVu/4Z3T9jPUt809vJEdkEfAjyemU9m5m+o/nbnmK97K2ko2M/SgE0kgPfglX+95dl6mqQtj/0sDdiEjgF3IyIW8P+vbStpC2UvS701kQBexiuvmrEno1xDMzOvAq4CT9yQhljHfraXpd6ayC7oe4B9I2KfiHgt1dUrbu5NWZIGzH6WBmzcn4Azc2NEnEF15YxJwDWZ+VDPKpM0MPazNHgDvRqSu62kri3KzINKF9HEXpa61tjL/iUsSZIKMIAlSSrAAJYkqQADWJKkAgxgSZIKMIAlSSrAAJYkqQADWJKkAgxgSZIKMIAlSSrAAJYkqQADWJKkAgxgSZIKMIAlSSrAAJYkqQADWJKkAgxgSZIKMIAlSSrAAJYkqQADWJKkAgxgSZIKMIAlSSrAAJYkqQADWJKkAgxgSZIKMIAlSSrAAJYkqYDtJnLniFgKrAc2ARsz86BeFCVp8OxnabAmFMC1IzNzdQ+WI6k8+1kaEHdBS5JUwEQDOIFbI2JRRCzoRUGSirGfpQGa6C7oIzJzWUTsBtwWEY9k5k9bZ6gb2WaWhl/bfraXpd6KzOzNgiIuADZk5hfbzNOblUlbv0UlT4Lq1M/2stS1xl4e9y7oiNgxInYauQ0cBTw43uVJKsd+lgZvIrugpwM3RcTIcr6VmX/fk6okDZr9LA3YuAM4M58EDuxhLZIKsZ+lwfNrSJIkFWAAS5JUgAEsSVIBBrAkSQUYwJIkFWAAS5JUQC+uhrRNmjJlSuPYvHnzGsdOP/30fpTT1sUXX9w4tm7durb33bBhQ+PYgw/6dxq05bOX7eVS/AQsSVIBBrAkSQUYwJIkFWAAS5JUgAEsSVIBBrAkSQVE5uCuq70lXcT7yCOPbDt+0UUXNY4ddthhvS6nmBdffLFx7I477mgc27hxY+PY+eef33ad7b5OsWrVqrb33Yo0XsR7GNjLWx57uZjGXvYTsCRJBRjAkiQVYABLklSAASxJUgEGsCRJBRjAkiQV4NeQGvzwhz9sO/6BD3xgQJVse5YsWdI49uUvf7lx7Gtf+1o/yinFryH1iL1cjr0M+DUkSZKGiwEsSVIBBrAkSQUYwJIkFWAAS5JUgAEsSVIB23WaISKuAY4BVmbmAfW0qcB3gVnAUuCEzPxV/8ocvLVr15YuYZu13377NY6deeaZjWPtrtpy9dVXT6imrcW22M/2cjn2cnvdfAK+Fpi/2bTzgNszc1/g9vp3ScPvWuxnaSh0DODM/CmwZrPJxwIL69sLgeN6W5akfrCfpeEx3mPA0zNzeX17BTC9R/VIGjz7WSqg4zHgTjIz2/1ZuohYACyY6Hok9V+7fraXpd4a7yfg5yNiBkD978qmGTPzqsw8aJj/rq20jeuqn+1lqbfGG8A3AyfVt08CftCbciQVYD9LBXQM4Ij4NvAz4M0R8WxEnAJcAsyLiMeA99W/Sxpy9rM0PLwcYYNJkya1HT/33HMbx2688cbGsaeeemrcNfXDvvvu23b81FNPHddy3/3udzeOHXjggeNaZidPPPFE49i8efPa3nfp0qU9rmbCvBxhj9jLFXu5GC9HKEnSMDGAJUkqwACWJKkAA1iSpAIMYEmSCjCAJUkqwK8hqS922223xrEVK1b0ZZ0rVzb+QTbmz9/8AkCvtHjx4h5XM2F+DWkIXHnllY1jp5122gArKcdenjC/hiRJ0jAxgCVJKsAAliSpAANYkqQCDGBJkgowgCVJKmC70gVo69Tuqwv98sgjjzSODeFXE7QF2Fa+atSOvdw/fgKWJKkAA1iSpAIMYEmSCjCAJUkqwACWJKkAA1iSpAL8GtIQmTlzZtvxadOmDaiS7kyePLlxbOHChX1Z59q1axvHrrjiir6sUxore7kze9lPwJIkFWEAS5JUgAEsSVIBBrAkSQUYwJIkFWAAS5JUgAEsSVIBHb8HHBHXAMcAKzPzgHraBcCpwKp6tvMz83/1q8h+2WmnnRrHdtlll7b33W+//RrHTj755HHVc/DBB7cdnz179riWuzW5/fbbG8duuummAVayZdpa+9le3vLYy919Ar4WmD/K9C9l5tz6Z4tqVmkbdi32szQUOgZwZv4UWDOAWiT1mf0sDY+JHAM+IyLuj4hrImJK00wRsSAi7o2IeyewLkn91bGf7WWpt8YbwF8FZgNzgeXApU0zZuZVmXlQZh40znVJ6q+u+tlelnprXAGcmc9n5qbMfBn4OnBIb8uSNCj2s1TGuAI4Ima0/PpB4MHelCNp0OxnqYxuvob0beA9wK4R8SzwJ8B7ImIukMBS4JP9K7F/zjjjjMaxiy++eICVqFtz585tHJszZ07j2DPPPNN2uZs2bRpXPRs3bhzX/UrZWvvZXt7y2MtdBHBmnjjK5Kv7UIukPrOfpeHhX8KSJKkAA1iSpAIMYEmSCjCAJUkqwACWJKmAyMzBrSxicCvrQrvHPsjtovJuvfXWcd1v/vzRrmvQE4uG+S9O2csaVltSL/sJWJKkAgxgSZIKMIAlSSrAAJYkqQADWJKkAgxgSZIK6Hgxhq3Z3Xff3Tj2jne8Y4CVDK/Vq1c3jt1www2NY4cffnjj2AEHHDChmvrhqKOOahxr9zzRcLCXO7OXh6+X/QQsSVIBBrAkSQUYwJIkFWAAS5JUgAEsSVIBBrAkSQUYwJIkFbBNX45w8uTJjWNTp05te99zzjmncez4449vHHv88ccbx5YtW9Z2nf1w+eWXtx1funRp41i7eqdNm9Y4tssuu7Rd54UXXtg4ts8++zSOHXrooY1jN910U9t13nnnnY1j119/fePY2rVr2y53Arwc4RjYy/byiC2pl/0ELElSAQawJEkFGMCSJBVgAEuSVIABLElSAQawJEkFdPwaUkTMBL4BTAcSuCozr4iIqcB3gVnAUuCEzPxVh2UN1VcX+uUtb3lL49jKlSsbx9asWdOPcrYq7b5SMmfOnMaxRx99tO1y+/gVhPHqy9eQetXP9rK9PFH2cnefgDcCZ2fm/sChwOkRsT9wHnB7Zu4L3F7/Lmm42c/SkOgYwJm5PDPvq2+vB5YAewDHAgvr2RYCx/WpRkk9Yj9Lw2O7scwcEbOAtwE/B6Zn5vJ6aAXVLq3R7rMAWDCBGiX1wVj72V6Weqvrk7Ai4vXADcBZmbmudSyrA8mjHhPKzKsy86Bh/rN60rZmPP1sL0u91VUAR8T2VM36zcy8sZ78fETMqMdnAM1nJEgaGvazNBw6BnBEBHA1sCQzL2sZuhk4qb59EvCD3pcnqZfsZ2l4dPM1pCOAu4AHgJfryedTHTf6HrAX8DTV1xbannu/rXx1QeqBfn0NqSf9bC9LXWvs5W36coTSEPNyhNLWwcsRSpI0TAxgSZIKMIAlSSrAAJYkqQADWJKkAgxgSZIKMIAlSSrAAJYkqQADWJKkAgxgSZIKMIAlSSrAAJYkqQADWJKkAgxgSZIKMIAlSSrAAJYkqQADWJKkAgxgSZIKMIAlSSrAAJYkqQADWJKkAgxgSZIKMIAlSSrAAJYkqQADWJKkAgxgSZIKMIAlSSqgYwBHxMyIuDMiHo6IhyLizHr6BRGxLCIW1z9H979cSeNlL0vDZbsu5tkInJ2Z90XETsCiiLitHvtSZn6xf+VJ6iF7WRoiHQM4M5cDy+vb6yNiCbBHvwuT1Fv2sjRcxnQMOCJmAW8Dfl5POiMi7o+IayJiSq+Lk9Qf9rJUXtcBHBGvB24AzsrMdcBXgdnAXKp31Zc23G9BRNwbEfdOvFxJE2UvS8MhMrPzTBHbA7cAP8rMy0YZnwXckpkHdFhO55VJAliUmQf1eqH2sjRwjb3czVnQAVwNLGlt2IiY0TLbB4EHJ1qlpP6xl6Xh0s1Z0IcDHwUeiIjF9bTzgRMjYi6QwFLgk32oT1Lv2MvSEOlqF3TPVuZuK6lbfdkF3Sv2stS18e+CliRJvWcAS5JUgAEsSVIBBrAkSQUYwJIkFWAAS5JUgAEsSVIBBrAkSQUYwJIkFWAAS5JUgAEsSVIBBrAkSQUYwJIkFdDN5Qh7aTXwdMvvu9bThoX1tDds9cDw1dSrevbuwTL6yV4eu2GryXra63svD/RyhK9aecS9w3TJNetpb9jqgeGradjqGZRhe9zDVg8MX03W094g6nEXtCRJBRjAkiQVUDqAryq8/s1ZT3vDVg8MX03DVs+gDNvjHrZ6YPhqsp72+l5P0WPAkiRtq0p/ApYkaZtUJIAjYn5E/HNEPB4R55WoYbN6lkbEAxGxOCLuLVTDNRGxMiIebJk2NSJui4jH6n+nFK7ngohYVm+nxRFx9ADrmRkRd0bEwxHxUEScWU8vso3a1FNsG5ViP79q/UPVy21qKvJcHbZe7lBTX7fRwHdBR8Qk4FFgHvAscA9wYmY+PNBCXlnTUuCgzCz2HbSIeBewAfhGZh5QT/sCsCYzL6lf2KZk5qcL1nMBsCEzvziIGjarZwYwIzPvi4idgEXAccDHKbCN2tRzAoW2UQn286jrH6peblPTBRR4rg5bL3eoqa/9XOIT8CHA45n5ZGb+BvgOcGyBOoZKZv4UWLPZ5GOBhfXthVRPiJL1FJOZyzPzvvr2emAJsAeFtlGberY19vNmhq2X29RUxLD1coea+qpEAO8BPNPy+7OUf+FK4NaIWBQRCwrX0mp6Zi6vb68AppcspnZGRNxf79Ia6G60ERExC3gb8HOGYBttVg8MwTYaIPu5O8Wfpw2KPleHrZdHqQn6uI08CatyRGa+HXg/cHq9u2aoZHWsoPQp618FZgNzgeXApYMuICJeD9wAnJWZ61rHSmyjUeopvo003P08JL0MhZ+rw9bLDTX1dRuVCOBlwMyW3/espxWTmcvqf1cCN1HtVhsGz9fHJkaOUawsWUxmPp+ZmzLzZeDrDHg7RcT2VM3xzcy8sZ5cbBuNVk/pbVSA/dydoeplKPtcHbZebqqp39uoRADfA+wbEftExGuBDwM3F6gDgIjYsT7oTkTsCBwFPNj+XgNzM3BSffsk4AcFaxlpihEfZIDbKSICuBpYkpmXtQwV2UZN9ZTcRoXYz90Zql6Gcs/VYevldjX1fRtl5sB/gKOpzpx8AvhMiRpaankT8E/1z0Ol6gG+TbWL4yWq42inANOA24HHgB8DUwvXcx3wAHA/VbPMGGA9R1DtkrofWFz/HF1qG7Wpp9g2KvVjP7+qhqHq5TY1FXmuDlsvd6ipr9vIv4QlSVIBnoQlSVIBBrAkSQUYwJIkFWAAS5JUgAEsSVIBBrAkSQUYwJIkFWAAS5JUwP8FlKEaL+DqPagAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# fig, ax = plt.subplots(1, 2, figsize=(8, 8))\n",
    "\n",
    "# ax[0].imshow(pairs[4][0].reshape(28, 28), cmap='gray')\n",
    "# ax[0].set_title('Correct class: 2')\n",
    "# ax[1].imshow(pairs[4][1].reshape(28, 28), cmap='gray')\n",
    "# ax[1].set_title('Misclassified as 6');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeAAAAD6CAYAAAB57pTcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAAZdElEQVR4nO3dffRcdX3g8fdHHi3CQgiN4SkpT9YYt4gBu1tgERSRAwbOrqyxFVAgFuEoCq0UWMGWtsAK6tlVdqFwAqhEBIUc3S0iPqDVKgnLM7YEGoSQkIQYnkTIw2f/uPfX/gi/OzOZ38x8J+T9OmdO5nc/d+79zM187mfu09zITCRJ0mC9rnQCkiRtimzAkiQVYAOWJKkAG7AkSQXYgCVJKsAGLElSATZgtRURP4yIk0vnIZUSEf8rIv7bOF5/YkT8pJc5rTf9/xsRJ4z6+8KIWBERSyNi94h4PiI262K6UyMiI2Lz3mYssAH3XER8MCLm1x/4JXVhHDgEeS2KiHeVzqMbEfE7EfHleoXyTETcUTonvTbUdfFyRExcb/j/qxvPVIDM/NPM/KsiSXYgM9+bmdcARMTuwJnAtMx8Y2b+KjPfkJlry2Y5tojYJyJuiYjlEbEyIm6NiDeVzmsQbMA9FBGfAr4A/A0wCdgd+DIws4tpveob5yb8LfQKYALw5vrfT5ZNR68x/wLMGvkjIt4K/E65dMZtd+DpzFxWOpEObQ/MA95Etd78BXBLyYQGJjN99OAB/DvgeeD9LcbZiqpBP1k/vgBsVccOAZ4APg0sBa4DLgBuBL4CPAucXM/nKmAJsBi4ENhs1DxOAR4CngMeBParp7UOeLHO8c8b8psJ3F3P6xHgiHr4D4GT6+d7At8HngZWAF8Fth81jU/XeT0H/BNwWD38AGB+Pe2ngMs6XK6/X79mu9L/xz5eew9gEXAecOeoYZ8DzgUSmFoPmwNcWD+fCHwbWAWsBH4MvK6O7QZ8E1he18j/rIefCPxk1Dy+CDxef7YXAAeNio1ZK8DW9brg6XredwKT6tgP6/XDu+o6X1fX+hxgav1eNq/HbVyHAJvV738F8Chw2ujXjrH8zq7XFSPrm2NHxfYCfgQ8U0/v6x3+n0yo57lj6c9Hvx9uAffOf6AqkG+1GOdc4A+BfYE/oCq080bF30j14ZsCzK6HzaRqwttTNbs5wBqqD/fbgMOpCo+IeD9V0z4e2A54H9U34Q8BvwKOzmpX1CXrJxYRBwDXAn9Wz+tgqpXTq0YF/hbYmWqLdLd6ntS7jU4H9s/MbYH3jJrGF4EvZuZ2VE38hlHzvjciPjjmEquW0WPAZ+td0PdFxH9uGFfqxj8C20XEm+vjpB+ganRNzqT6srwT1RbbOUDWr/021ed1KrALMLdhGndSrQcmAF8DvhERW9explo5gap57gbsCPwpVbP9V5n5PeC9wJN1rZ84xrzn0LAOofoCf1Q9fAbwX1osB6ia70F1Xp8FvhIRk+vYXwHfBXYAdgX+R5tpjTgYWJqZT3c4/kbLBtw7OwIrMnNNi3H+GPjLzFyWmcupPrAfGhVfB5yfmS9l5khh/Swzb87MdVRN9UjgjMx8IatdTJ+nWmFAVUSXZOadWVmYmY91mP9JwNWZeVtmrsvMxZn5y/VHqqd5W53jcuAy4D/V4bVUW/nTImKLzFyUmY/UsdXAXhExMTOfz8x/HDXNf5+ZX2vIa1dgOtW36J2pGvw1EfHmDt+X1InrqL64vptqD9LiFuOuBiYDUzJzdWb+OKtNtwOoPqN/VtfnbzNzzBOvMvMrmfl0Zq7JzEup6uZNo6Y/Vq2splrP7JWZazNzQWY+uyFvMiIm0Xodchzwhcx8PDNXUn3ZbpSZ38jMJ+t1xteBh+vlMJLvFGDnVstivfx2Bb4EfGpD3tfGygbcO08DE9scp92Z6tvxiMfqYSOWZ+Zv13vN46OeTwG2AJZExKqIWAX8b+B36/huVN9Iu9HRayNiUkTMjYjFEfEs1ZbCRKiaM3AG1Rbxsnq8kfd3ErAP8MuIuDMijuowrxepCvnCzHw5M38E/IDqW7vUK9cBH6TaVXxtm3H/O7AQ+G5EPBoRZ9fDdwMea/MlHICIOCsiHqpPKlxFtQU5ciJYU61cB9wKzI2IJyPikojYovO3CLRfh+zMK9c5Lb/AR8TxEXH3qGlNH/U+/pxqj9kvIuKBiPhIm2ntRLXF/OXMvH7D3tbGyQbcOz8DXgKOaTHOk1QFMGL3etiIsW5NNXrY4/U8Jmbm9vVju8x8y6j4ng3zbnfbq1avHe1v6mm9td5F9idURVbNJPNrmXkg1ftM4OJ6+MOZOYuq0C8GboyIbTqY371jDPMWXuqpek/Rv1BtHX6zzbjPZeaZmbkH1WGeT0XEYVQ1tHu7kyUj4iCq5nQcsENmbk+1hyfq6Y9ZK/XW9mczcxrwH6l2FR+/gW+13TpkCdUXiRG7t3gfU4ArqfZK7Vi/j/tHvY+lmXlKZu4MfBT4ckTs1TCtHaia77zM/OsNfE8bLRtwj2TmM8BngC9FxDH1pTNbRMR7I2LkmOv1wHkRsVN92cNnaH2saf15LKH6kF4aEdtFxOsiYs+IGNkF/HfAWRHx9qjsVRcJVCdz7NFi8lcBH46Iw+rp7hIRvz/GeNtSndzxTETsQnXMGKiOAUfEoRGxFfBb/u1kECLiTyJip3pX+qr6Jes6eNt3UB2//ouI2Dwi/gh4J9WWgNRLJwGHZuYLrUaKiKPq2gqqxrmW6rP8C6oGdlFEbBMRW9ef1/VtS3UMdjmweUR8hurw0sj0x6yViHhnRLy1Ptb8LNWeoU5q6F91sA65Afh4ROxaN8WzGycG21B9GV5e5/1hqi3gkffx/nqXMsCv63FflW9EbEdVz/+Qma3m95pjA+6h+ljOp6hOrFpO9W3zdODmepQLqc5uvBe4D7irHrYhjge2pDrj8NdUJ2hNruf/DeCvqU7qeK6e74T6dX9L1fxXRcRZY+T+C+DDVMeDnqE6e3HK+uNRHbferx7nO7xya2Er4CKqMx6XUn2D/4s6dgTwQEQ8T3WSyQdGjnPXu6f+eKw3m5mrqU5EO7Ke55XA8WMdn5bGIzMfycz5HYy6N/A9qi+iP6PaZfqDrK6zPZrq5KZfUZ2o9V/HeP2twN8D/0y1i/e3vHK3b1OtvJGq3p+lOk79I6rd0huqcR1CVV+3AvdQrZ8a9wZk5oPApVTL4CngrcA/jBplf+Dn9fuYB3wiMx8dY1LH1uN+OKrfTxh5NG59v1ZEde6AJEkaJLeAJUkqwAYsSVIBNmBJkgqwAUuSVIANWJKkAsZ1d52IOILqNPnNgL/LzIvajO8p11JnVmTmToOc4YbUs7UsdayxlrveAq4vBv8S1Q9/TwNmRcS0bqcn6RU6/Q3vnrCepb5prOXx7II+AFiYmY9m5stUd/3Y4PveShoK1rM0YONpwLvwyl9veaIeJmnjYz1LAzauY8CdiIjZ/Nu9bSVtpKxlqbfG04AX88q7ZuzKGPfQzMwrgCvAEzekIda2nq1lqbfGswv6TmDviPi9iNiS6obO83qTlqQBs56lAet6Czgz10TE6VR3ztgMuDozH+hZZpIGxnqWBm+gd0Nyt5XUsQWZOaN0Ek2sZaljjbXsL2FJklSADViSpAJswJIkFWADliSpABuwJEkF9P2XsNQ7++23X2Ps8ssvb4zdd999jbHTTjut5Txfeuml9olJ2iDWssAtYEmSirABS5JUgA1YkqQCbMCSJBVgA5YkqQAbsCRJBdiAJUkqwOuANyLvec97GmNvf/vbG2Obb97839wqBl47KPWDtSxwC1iSpCJswJIkFWADliSpABuwJEkF2IAlSSrABixJUgFehjREpkyZ0jJ+8sknN8YyszF28cUXN8ZeeOGF9olJ2iDWsjrhFrAkSQXYgCVJKsAGLElSATZgSZIKsAFLklSADViSpALGdRlSRCwCngPWAmsyc0YvktpUnXXWWS3jrS5tWL58eWPshhtu6DonbTqs596xltWJXlwH/M7MXNGD6Ugqz3qWBsRd0JIkFTDeBpzAdyNiQUTM7kVCkoqxnqUBGu8u6AMzc3FE/C5wW0T8MjPvGD1CXcgWszT8WtaztSz11ri2gDNzcf3vMuBbwAFjjHNFZs7whA5puLWrZ2tZ6q2uG3BEbBMR2448Bw4H7u9VYpIGx3qWBm88u6AnAd+KiJHpfC0z/74nWW2iZs2a1fVr586d28NMtAmynnvIWlYnum7Amfko8Ac9zEVSIdazNHhehiRJUgE2YEmSCrABS5JUgA1YkqQCbMCSJBVgA5YkqYDIzMHNLGJwMxtSEyZMaIytWNH6JjR33XVXY+yggw5qjL344ovtE9OwWTDMvzhlLVvL6lhjLbsFLElSATZgSZIKsAFLklSADViSpAJswJIkFWADliSpgPHcjlBdOPfccxtj7S4JW7hwYWPMyxOkwbKWNV5uAUuSVIANWJKkAmzAkiQVYAOWJKkAG7AkSQXYgCVJKsDLkAbs8MMP7/q1w3Z5wvbbb98Y23LLLRtjL7/8csvprlq1qsuMpMGxlq3l8XILWJKkAmzAkiQVYAOWJKkAG7AkSQXYgCVJKsAGLElSAW0vQ4qIq4GjgGWZOb0eNgH4OjAVWAQcl5m/7l+aApgzZ87A5/mOd7yjMXbLLbc0xiZOnNgYW758ect57r///o2xJ554ouVr1Zr1PBysZWsZOtsCngMcsd6ws4HbM3Nv4Pb6b0nDbw7WszQU2jbgzLwDWLne4JnANfXza4BjepuWpH6wnqXh0e0x4EmZuaR+vhSY1KN8JA2e9SwVMO6foszMjIhsikfEbGD2eOcjqf9a1bO1LPVWt1vAT0XEZID632VNI2bmFZk5IzNndDkvSf3VUT1by1JvdduA5wEn1M9PAJpPoZM07KxnqYC2DTgirgd+BrwpIp6IiJOAi4B3R8TDwLvqvyUNOetZGh5tjwFn5qyG0GE9zkVD6Dvf+U5jrNUtzFrZaaedWsZPOeWUxtj555/f1TxVsZ43Xdby8PGXsCRJKsAGLElSATZgSZIKsAFLklSADViSpAJswJIkFTDun6JU70TEwOc5d+7clvEdd9yxMbZq1arG2NKlSxtj++yzT8t5nnfeeY0xL13QxsBarljLrbkFLElSATZgSZIKsAFLklSADViSpAJswJIkFWADliSpAC9DGrBHHnmkMTZt2rQBZlJ5y1ve0jK+bt26xtjHPvaxxthNN93UGDv11FNbzvPSSy9tjB1zzDGNsZtvvrnldKVespat5fFyC1iSpAJswJIkFWADliSpABuwJEkF2IAlSSrABixJUgFehjRgF154YWPs6KOPbvna17/+9V3N84wzzmiMtbubyT333NMYu/HGGxtjq1ev7mqa7UyePLnr10q9ZC1by+PlFrAkSQXYgCVJKsAGLElSATZgSZIKsAFLklSADViSpAJswJIkFdD2OuCIuBo4CliWmdPrYRcApwDL69HOycz/068kX0vmz5/f9WtPPPHExtj3v//9xtj73ve+xthmm23Wcp5XXnllY6zV9YEaTtZz71jLGq9OtoDnAEeMMfzzmblv/bBYpY3DHKxnaSi0bcCZeQewcgC5SOoz61kaHuM5Bnx6RNwbEVdHxA5NI0XE7IiYHxHd76+R1G9t69lalnqr2wZ8ObAnsC+wBLi0acTMvCIzZ2TmjC7nJam/Oqpna1nqra4acGY+lZlrM3MdcCVwQG/TkjQo1rNURlcNOCJG38biWOD+3qQjadCsZ6mMTi5Duh44BJgYEU8A5wOHRMS+QAKLgI/2L8VNx9y5c1vGjzzyyMZYq9ufHXzwwV3n1A/Tp08vncImy3oeDGtZnWjbgDNz1hiDr+pDLpL6zHqWhoe/hCVJUgE2YEmSCrABS5JUgA1YkqQCbMCSJBXQ9ixoDc7Kla1/one77bZrjF177bWNsUcffbQxtueee7ac5+WXX94y3mTbbbdtjH384x9v+dqIaIy97nV+Z9Tws5Yr1nJrLgFJkgqwAUuSVIANWJKkAmzAkiQVYAOWJKkAG7AkSQV4GdIQmTdvXsv4qaee2hjbaqutGmN77LFHYywzW85zr732aoy99NJLjbFPfvKTXeXTLqeFCxe2fK00DKzl9jlZy24BS5JUhA1YkqQCbMCSJBVgA5YkqQAbsCRJBdiAJUkqwAYsSVIB0e7asZ7OLGJwM9sIbb5568uyL7nkksZYu9uCdes3v/lNY2zt2rWNsVa3MGvnjjvuaIwdeuihXU93I7MgM2eUTqKJtdyatVyxloEWtewWsCRJBdiAJUkqwAYsSVIBNmBJkgqwAUuSVIANWJKkAtrejjAidgOuBSYBCVyRmV+MiAnA14GpwCLguMz8df9Sfe1bs2ZNy/hll13WGJs+fXpjbDyn+2+zzTaNsVaXsK1evbox9tOf/rTlPD/ykY+0T0xdsZ4Hw1quWMutdbIFvAY4MzOnAX8InBYR04Czgdszc2/g9vpvScPNepaGRNsGnJlLMvOu+vlzwEPALsBM4Jp6tGuAY/qUo6QesZ6l4dF2F/RoETEVeBvwc2BSZi6pQ0updmmN9ZrZwOxx5CipDza0nq1lqbc6PgkrIt4A3ASckZnPjo5ldQBhzIMImXlFZs4Y5p/VkzY13dSztSz1VkcNOCK2oCrWr2bmN+vBT0XE5Do+GVjWnxQl9ZL1LA2Htg04IgK4CngoM0efujcPOKF+fgJwS+/Tk9RL1rM0PNreDSkiDgR+DNwHrKsHn0N13OgGYHfgMarLFla2mZZ3UOmTVndfOeSQQxpjM2fObDndgw8+uDG29dZbN8Zuuummxtg555zTcp4C+nQ3pF7Vs7UMEydObIytWLGi6+lay685jbXc9iSszPwJEA3hw8aTlaTBsp6l4eEvYUmSVIANWJKkAmzAkiQVYAOWJKkAG7AkSQW0vQyppzPz0gWpU325DKlXrGWpY4217BawJEkF2IAlSSrABixJUgE2YEmSCrABS5JUgA1YkqQCbMCSJBVgA5YkqQAbsCRJBdiAJUkqwAYsSVIBNmBJkgqwAUuSVIANWJKkAmzAkiQVYAOWJKkAG7AkSQXYgCVJKsAGLElSATZgSZIKsAFLklRA2wYcEbtFxA8i4sGIeCAiPlEPvyAiFkfE3fXjyP6nK6lb1rI0XDbvYJw1wJmZeVdEbAssiIjb6tjnM/Nz/UtPUg9Zy9IQaduAM3MJsKR+/lxEPATs0u/EJPWWtSwNlw06BhwRU4G3AT+vB50eEfdGxNURsUOvk5PUH9ayVF7HDTgi3gDcBJyRmc8ClwN7AvtSfau+tOF1syNifkTMH3+6ksbLWpaGQ2Rm+5EitgC+DdyamZeNEZ8KfDszp7eZTvuZSQJYkJkzej1Ra1kauMZa7uQs6ACuAh4aXbARMXnUaMcC9483S0n9Yy1Lw6WTs6D/CPgQcF9E3F0POweYFRH7AgksAj7ah/wk9Y61LA2RjnZB92xm7raSOtWXXdC9Yi1LHet+F7QkSeo9G7AkSQXYgCVJKsAGLElSATZgSZIKsAFLklSADViSpAJswJIkFWADliSpABuwJEkF2IAlSSrABixJUgE2YEmSCujkdoS9tAJ4bNTfE+thw8J8Whu2fGD4cupVPlN6MI1+spY33LDlZD6t9b2WB3o7wlfNPGL+MN1yzXxaG7Z8YPhyGrZ8BmXY3vew5QPDl5P5tDaIfNwFLUlSATZgSZIKKN2Aryg8//WZT2vDlg8MX07Dls+gDNv7HrZ8YPhyMp/W+p5P0WPAkiRtqkpvAUuStEkq0oAj4oiI+KeIWBgRZ5fIYb18FkXEfRFxd0TML5TD1RGxLCLuHzVsQkTcFhEP1//uUDifCyJicb2c7o6IIweYz24R8YOIeDAiHoiIT9TDiyyjFvkUW0alWM+vmv9Q1XKLnIp8Voetltvk1NdlNPBd0BGxGfDPwLuBJ4A7gVmZ+eBAE3llTouAGZlZ7Bq0iDgYeB64NjOn18MuAVZm5kX1im2HzPx0wXwuAJ7PzM8NIof18pkMTM7MuyJiW2ABcAxwIgWWUYt8jqPQMirBeh5z/kNVyy1yuoACn9Vhq+U2OfW1nktsAR8ALMzMRzPzZWAuMLNAHkMlM+8AVq43eCZwTf38GqoPRMl8isnMJZl5V/38OeAhYBcKLaMW+WxqrOf1DFstt8ipiGGr5TY59VWJBrwL8Piov5+g/Iorge9GxIKImF04l9EmZeaS+vlSYFLJZGqnR8S99S6tge5GGxERU4G3AT9nCJbRevnAECyjAbKeO1P8c9qg6Gd12Gp5jJygj8vIk7AqB2bmfsB7gdPq3TVDJatjBaVPWb8c2BPYF1gCXDroBCLiDcBNwBmZ+ezoWIllNEY+xZeRhrueh6SWofBnddhquSGnvi6jEg14MbDbqL93rYcVk5mL63+XAd+i2q02DJ6qj02MHKNYVjKZzHwqM9dm5jrgSga8nCJiC6ri+GpmfrMeXGwZjZVP6WVUgPXcmaGqZSj7WR22Wm7Kqd/LqEQDvhPYOyJ+LyK2BD4AzCuQBwARsU190J2I2AY4HLi/9asGZh5wQv38BOCWgrmMFMWIYxngcoqIAK4CHsrMy0aFiiyjpnxKLqNCrOfODFUtQ7nP6rDVcquc+r6MMnPgD+BIqjMnHwHOLZHDqFz2AO6pHw+Uyge4nmoXx2qq42gnATsCtwMPA98DJhTO5zrgPuBeqmKZPMB8DqTaJXUvcHf9OLLUMmqRT7FlVOphPb8qh6Gq5RY5FfmsDlstt8mpr8vIX8KSJKkAT8KSJKkAG7AkSQXYgCVJKsAGLElSATZgSZIKsAFLklSADViSpAJswJIkFfD/ASICNzyV6kKPAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# fig, ax = plt.subplots(1, 2, figsize=(8, 8))\n",
    "\n",
    "# ax[0].imshow(pairs[58][0].reshape(28, 28), cmap='gray')\n",
    "# ax[0].set_title('Correct class: 6')\n",
    "# ax[1].imshow(pairs[58][1].reshape(28, 28), cmap='gray')\n",
    "# ax[1].set_title('Misclassified as 2');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converted 00_core.ipynb.\n",
      "Converted 01_Representation.ipynb.\n",
      "Converted 02_SymbolicInstance.ipynb.\n",
      "Converted index.ipynb.\n"
     ]
    }
   ],
   "source": [
    "#hide\n",
    "from nbdev.export import notebook2script\n",
    "notebook2script()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
